

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>geoslurp.datapull package &mdash; geoslurp v1.1.0 documentation</title>
  

  
  
    <link rel="shortcut icon" href="../_static/favicon.ico"/>
  
  
  

  

  
  
    

  

  
  
    <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  

  

  
        <link rel="index" title="Index"
              href="../genindex.html"/>
        <link rel="search" title="Search" href="../search.html"/>
    <link rel="top" title="geoslurp v1.1.0 documentation" href="../index.html"/>
        <link rel="up" title="geoslurp package" href="geoslurp.html"/>
        <link rel="next" title="geoslurp.dataset package" href="geoslurp.dataset.html"/>
        <link rel="prev" title="geoslurp.config package" href="geoslurp.config.html"/> 

  
  <script src="../_static/js/modernizr.min.js"></script>

</head>

<body class="wy-body-for-nav" role="document">

   
  <div class="wy-grid-for-nav">

    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search">
          

          
            <a href="../index.html" class="icon icon-home"> geoslurp
          

          
          </a>

          
            
            
              <div class="version">
                1.1.0
              </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">Contents:</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../introduction.html">An introduction to Geoslurp</a></li>
<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation of the geoslurp package and setting up the PostgreSQL instance</a></li>
<li class="toctree-l1"><a class="reference internal" href="../confsettings.html">Configuration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../examples.html">Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference.html">Reference Documentation</a></li>
</ul>

            
          
        </div>
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../index.html">geoslurp</a>
        
      </nav>


      
      <div class="wy-nav-content">
        <div class="rst-content">
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html">Docs</a> &raquo;</li>
        
          <li><a href="../reference.html">Reference Documentation</a> &raquo;</li>
        
          <li><a href="geoslurp.html">geoslurp package</a> &raquo;</li>
        
      <li>geoslurp.datapull package</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
            <a href="../_sources/reference/geoslurp.datapull.rst.txt" rel="nofollow"> View page source</a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  
<style>
/* CSS overrides for sphinx_rtd_theme */

/* 24px margin */
.nbinput.nblast,
.nboutput.nblast {
    margin-bottom: 19px;  /* padding has already 5px */
}

/* ... except between code cells! */
.nblast + .nbinput {
    margin-top: -19px;
}

.admonition > p:before {
    margin-right: 4px;  /* make room for the exclamation icon */
}
</style>
<div class="section" id="module-geoslurp.datapull">
<span id="geoslurp-datapull-package"></span><h1>geoslurp.datapull package<a class="headerlink" href="#module-geoslurp.datapull" title="Permalink to this headline">¶</a></h1>
<div class="section" id="submodules">
<h2>Submodules<a class="headerlink" href="#submodules" title="Permalink to this headline">¶</a></h2>
</div>
<div class="section" id="module-geoslurp.datapull.crawler">
<span id="geoslurp-datapull-crawler-module"></span><h2>geoslurp.datapull.crawler module<a class="headerlink" href="#module-geoslurp.datapull.crawler" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="geoslurp.datapull.crawler.CrawlerBase">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.crawler.</code><code class="descname">CrawlerBase</code><span class="sig-paren">(</span><em>url</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/crawler.html#CrawlerBase"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.crawler.CrawlerBase" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">abc.ABC</span></code></p>
<dl class="method">
<dt id="geoslurp.datapull.crawler.CrawlerBase.parallelDownload">
<code class="descname">parallelDownload</code><span class="sig-paren">(</span><em>outdir</em>, <em>check=False</em>, <em>maxconn=8</em>, <em>gzip=False</em>, <em>continueonError=False</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/crawler.html#CrawlerBase.parallelDownload"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.crawler.CrawlerBase.parallelDownload" title="Permalink to this definition">¶</a></dt>
<dd><p>Download uris in parallel
:param direc: directory to download to
:param check: Only download when newer or non-existent (default to False)
:param maxconn: amount of parallel downloads to execute
:param continueOnError (bool): keep trying</p>
</dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.crawler.CrawlerBase.rooturl">
<code class="descname">rooturl</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.crawler.CrawlerBase.rooturl" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.crawler.CrawlerBase.uris">
<code class="descname">uris</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/crawler.html#CrawlerBase.uris"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.crawler.CrawlerBase.uris" title="Permalink to this definition">¶</a></dt>
<dd><p>Generator which returns uri’s to requested datasets</p>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="module-geoslurp.datapull.ftp">
<span id="geoslurp-datapull-ftp-module"></span><h2>geoslurp.datapull.ftp module<a class="headerlink" href="#module-geoslurp.datapull.ftp" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="geoslurp.datapull.ftp.Crawler">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.ftp.</code><code class="descname">Crawler</code><span class="sig-paren">(</span><em>url</em>, <em>pattern='.*'</em>, <em>followpattern='.*'</em>, <em>auth=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/ftp.html#Crawler"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.ftp.Crawler" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <a class="reference internal" href="#geoslurp.datapull.crawler.CrawlerBase" title="geoslurp.datapull.crawler.CrawlerBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">geoslurp.datapull.crawler.CrawlerBase</span></code></a></p>
<p>Crawler for ftp directories</p>
<dl class="method">
<dt id="geoslurp.datapull.ftp.Crawler.ls">
<code class="descname">ls</code><span class="sig-paren">(</span><em>subdirs=''</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/ftp.html#Crawler.ls"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.ftp.Crawler.ls" title="Permalink to this definition">¶</a></dt>
<dd><p>List directories and files (generator)</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.ftp.Crawler.uris">
<code class="descname">uris</code><span class="sig-paren">(</span><em>check=False</em>, <em>subdirs=''</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/ftp.html#Crawler.uris"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.ftp.Crawler.uris" title="Permalink to this definition">¶</a></dt>
<dd><p>Generate a list files in a directory and return a list of uri</p>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="geoslurp.datapull.ftp.Uri">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.ftp.</code><code class="descname">Uri</code><span class="sig-paren">(</span><em>url</em>, <em>lastmod=None</em>, <em>subdirs=''</em>, <em>auth=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/ftp.html#Uri"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.ftp.Uri" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <a class="reference internal" href="#geoslurp.datapull.uri.UriBase" title="geoslurp.datapull.uri.UriBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">geoslurp.datapull.uri.UriBase</span></code></a></p>
</dd></dl>

</div>
<div class="section" id="module-geoslurp.datapull.geodesyunr">
<span id="geoslurp-datapull-geodesyunr-module"></span><h2>geoslurp.datapull.geodesyunr module<a class="headerlink" href="#module-geoslurp.datapull.geodesyunr" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="geoslurp.datapull.geodesyunr.Crawler">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.geodesyunr.</code><code class="descname">Crawler</code><span class="sig-paren">(</span><em>catalogfile</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/geodesyunr.html#Crawler"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.geodesyunr.Crawler" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <a class="reference internal" href="#geoslurp.datapull.crawler.CrawlerBase" title="geoslurp.datapull.crawler.CrawlerBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">geoslurp.datapull.crawler.CrawlerBase</span></code></a></p>
<p>Crawl the gps tenv3 data on geodesy.unr.edu</p>
<dl class="method">
<dt id="geoslurp.datapull.geodesyunr.Crawler.uris">
<code class="descname">uris</code><span class="sig-paren">(</span><em>refresh=True</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/geodesyunr.html#Crawler.uris"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.geodesyunr.Crawler.uris" title="Permalink to this definition">¶</a></dt>
<dd><p>List uris of available gps final data in tenv3 format</p>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="geoslurp.datapull.geodesyunr.Uri">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.geodesyunr.</code><code class="descname">Uri</code><span class="sig-paren">(</span><em>indict</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/geodesyunr.html#Uri"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.geodesyunr.Uri" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <a class="reference internal" href="#geoslurp.datapull.uri.UriBase" title="geoslurp.datapull.uri.UriBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">geoslurp.datapull.uri.UriBase</span></code></a></p>
<p>derived class which additionally holds info from the inventory</p>
</dd></dl>

</div>
<div class="section" id="module-geoslurp.datapull.github">
<span id="geoslurp-datapull-github-module"></span><h2>geoslurp.datapull.github module<a class="headerlink" href="#module-geoslurp.datapull.github" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="geoslurp.datapull.github.Crawler">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.github.</code><code class="descname">Crawler</code><span class="sig-paren">(</span><em>reponame</em>, <em>commitsha=None</em>, <em>filter=&lt;geoslurp.datapull.github.GithubFilter object&gt;</em>, <em>followfilt=&lt;geoslurp.datapull.github.GithubFilter object&gt;</em>, <em>oauthtoken=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/github.html#Crawler"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.github.Crawler" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <a class="reference internal" href="#geoslurp.datapull.crawler.CrawlerBase" title="geoslurp.datapull.crawler.CrawlerBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">geoslurp.datapull.crawler.CrawlerBase</span></code></a></p>
<p>Crawls a github repository fixed to a certain commit</p>
<dl class="method">
<dt id="geoslurp.datapull.github.Crawler.getSubTree">
<code class="descname">getSubTree</code><span class="sig-paren">(</span><em>url</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/github.html#Crawler.getSubTree"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.github.Crawler.getSubTree" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.github.Crawler.treeitems">
<code class="descname">treeitems</code><span class="sig-paren">(</span><em>rootelem=None</em>, <em>depth=10</em>, <em>dirpath=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/github.html#Crawler.treeitems"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.github.Crawler.treeitems" title="Permalink to this definition">¶</a></dt>
<dd><p>generator which recursively list all elements in a git tree</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.github.Crawler.uris">
<code class="descname">uris</code><span class="sig-paren">(</span><em>depth=10</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/github.html#Crawler.uris"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.github.Crawler.uris" title="Permalink to this definition">¶</a></dt>
<dd><p>Construct Uris from tree nodes</p>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="geoslurp.datapull.github.GithubFilter">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.github.</code><code class="descname">GithubFilter</code><span class="sig-paren">(</span><em>regexdict={'type': 'blob'}</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/github.html#GithubFilter"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.github.GithubFilter" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p>
<p>Filter used for testing a certain dict element</p>
<dl class="method">
<dt id="geoslurp.datapull.github.GithubFilter.isValid">
<code class="descname">isValid</code><span class="sig-paren">(</span><em>elem</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/github.html#GithubFilter.isValid"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.github.GithubFilter.isValid" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns True if all of the regex criteria match the elem</p>
</dd></dl>

</dd></dl>

<dl class="function">
<dt id="geoslurp.datapull.github.cachedGithubCatalogue">
<code class="descclassname">geoslurp.datapull.github.</code><code class="descname">cachedGithubCatalogue</code><span class="sig-paren">(</span><em>reponame</em>, <em>cachedir='.'</em>, <em>commitsha=None</em>, <em>gfilter=&lt;geoslurp.datapull.github.GithubFilter object&gt;</em>, <em>gfollowfilter=&lt;geoslurp.datapull.github.GithubFilter object&gt;</em>, <em>depth=2</em>, <em>ghtoken=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/github.html#cachedGithubCatalogue"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.github.cachedGithubCatalogue" title="Permalink to this definition">¶</a></dt>
<dd><p>Caches the result of a github result for later reuse</p>
</dd></dl>

</div>
<div class="section" id="module-geoslurp.datapull.http">
<span id="geoslurp-datapull-http-module"></span><h2>geoslurp.datapull.http module<a class="headerlink" href="#module-geoslurp.datapull.http" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="geoslurp.datapull.http.Uri">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.http.</code><code class="descname">Uri</code><span class="sig-paren">(</span><em>url</em>, <em>lastmod=None</em>, <em>auth=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/http.html#Uri"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.http.Uri" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <a class="reference internal" href="#geoslurp.datapull.uri.UriBase" title="geoslurp.datapull.uri.UriBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">geoslurp.datapull.uri.UriBase</span></code></a></p>
</dd></dl>

</div>
<div class="section" id="module-geoslurp.datapull.icgem">
<span id="geoslurp-datapull-icgem-module"></span><h2>geoslurp.datapull.icgem module<a class="headerlink" href="#module-geoslurp.datapull.icgem" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="geoslurp.datapull.icgem.Crawler">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.icgem.</code><code class="descname">Crawler</code><a class="reference internal" href="../_modules/geoslurp/datapull/icgem.html#Crawler"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.icgem.Crawler" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <a class="reference internal" href="#geoslurp.datapull.crawler.CrawlerBase" title="geoslurp.datapull.crawler.CrawlerBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">geoslurp.datapull.crawler.CrawlerBase</span></code></a></p>
<p>Crawl icgem static fields</p>
<dl class="method">
<dt id="geoslurp.datapull.icgem.Crawler.uris">
<code class="descname">uris</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/icgem.html#Crawler.uris"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.icgem.Crawler.uris" title="Permalink to this definition">¶</a></dt>
<dd><p>List uris of available static models</p>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="geoslurp.datapull.icgem.Uri">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.icgem.</code><code class="descname">Uri</code><span class="sig-paren">(</span><em>url</em>, <em>lastmod=None</em>, <em>name=None</em>, <em>ref=None</em>, <em>nmax=None</em>, <em>year=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/icgem.html#Uri"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.icgem.Uri" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <a class="reference internal" href="#geoslurp.datapull.uri.UriBase" title="geoslurp.datapull.uri.UriBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">geoslurp.datapull.uri.UriBase</span></code></a></p>
<p>Holds an uri to an icgem static field</p>
</dd></dl>

</div>
<div class="section" id="module-geoslurp.datapull.motu">
<span id="geoslurp-datapull-motu-module"></span><h2>geoslurp.datapull.motu module<a class="headerlink" href="#module-geoslurp.datapull.motu" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="geoslurp.datapull.motu.MotuOpts">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.motu.</code><code class="descname">MotuOpts</code><span class="sig-paren">(</span><em>moturoot</em>, <em>service</em>, <em>product</em>, <em>auth</em>, <em>btdbox</em>, <em>fout</em>, <em>cache</em>, <em>variables=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/motu.html#MotuOpts"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p>
<p>A class which mimics the options from argparse as used by the motuclient command line program</p>
<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.auth_mode">
<code class="descname">auth_mode</code><em class="property"> = 'cas'</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.auth_mode" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.block_size">
<code class="descname">block_size</code><em class="property"> = 12001</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.block_size" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.btdbox">
<code class="descname">btdbox</code><em class="property"> = &lt;geoslurp.tools.Bounds.BtdBox object&gt;</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.btdbox" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.cache">
<code class="descname">cache</code><em class="property"> = '.'</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.cache" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.console_mode">
<code class="descname">console_mode</code><em class="property"> = False</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.console_mode" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.date_max">
<code class="descname">date_max</code><em class="property"> = '9999-12-31 23:59:59'</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.date_max" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.date_min">
<code class="descname">date_min</code><em class="property"> = '1-01-01 00:00:00'</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.date_min" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.depth_max">
<code class="descname">depth_max</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.depth_max" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.depth_min">
<code class="descname">depth_min</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.depth_min" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.describe">
<code class="descname">describe</code><em class="property"> = False</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.describe" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.extraction_geographic">
<code class="descname">extraction_geographic</code><em class="property"> = True</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.extraction_geographic" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.extraction_vertical">
<code class="descname">extraction_vertical</code><em class="property"> = False</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.extraction_vertical" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.motu.MotuOpts.fullname">
<code class="descname">fullname</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/motu.html#MotuOpts.fullname"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.fullname" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.latitude_max">
<code class="descname">latitude_max</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.latitude_max" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.latitude_min">
<code class="descname">latitude_min</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.latitude_min" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.longitude_max">
<code class="descname">longitude_max</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.longitude_max" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.longitude_min">
<code class="descname">longitude_min</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.longitude_min" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.motu">
<code class="descname">motu</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.motu" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.out_dir">
<code class="descname">out_dir</code><em class="property"> = '.'</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.out_dir" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.out_name">
<code class="descname">out_name</code><em class="property"> = 'dataset.nc'</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.out_name" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.outputWritten">
<code class="descname">outputWritten</code><em class="property"> = 'netcdf'</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.outputWritten" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.product_id">
<code class="descname">product_id</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.product_id" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.proxy_server">
<code class="descname">proxy_server</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.proxy_server" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.pwd">
<code class="descname">pwd</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.pwd" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.service_id">
<code class="descname">service_id</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.service_id" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.size">
<code class="descname">size</code><em class="property"> = False</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.size" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.socket_timeout">
<code class="descname">socket_timeout</code><em class="property"> = 515</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.socket_timeout" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.sync">
<code class="descname">sync</code><em class="property"> = False</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.sync" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.motu.MotuOpts.syncbtdbox">
<code class="descname">syncbtdbox</code><span class="sig-paren">(</span><em>bbox=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/motu.html#MotuOpts.syncbtdbox"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.syncbtdbox" title="Permalink to this definition">¶</a></dt>
<dd><p>Sets the internal btdbox and synchronize the corresponding motu variables</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.motu.MotuOpts.syncfilename">
<code class="descname">syncfilename</code><span class="sig-paren">(</span><em>fout</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/motu.html#MotuOpts.syncfilename"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.syncfilename" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.user">
<code class="descname">user</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.user" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.user_agent">
<code class="descname">user_agent</code><em class="property"> = 'motu-api-client'</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.user_agent" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuOpts.variable">
<code class="descname">variable</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuOpts.variable" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

</dd></dl>

<dl class="class">
<dt id="geoslurp.datapull.motu.MotuRecursive">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.motu.</code><code class="descname">MotuRecursive</code><span class="sig-paren">(</span><em>mopts</em>, <em>keepfiles=False</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/motu.html#MotuRecursive"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.motu.MotuRecursive" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p>
<p>Class which recursively downloads netcdf files within the 1GB limit using motu and patches them together</p>
<dl class="method">
<dt id="geoslurp.datapull.motu.MotuRecursive.download">
<code class="descname">download</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/motu.html#MotuRecursive.download"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.motu.MotuRecursive.download" title="Permalink to this definition">¶</a></dt>
<dd><p>Download file</p>
</dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.MotuRecursive.keepfiles">
<code class="descname">keepfiles</code><em class="property"> = False</em><a class="headerlink" href="#geoslurp.datapull.motu.MotuRecursive.keepfiles" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

</dd></dl>

<dl class="class">
<dt id="geoslurp.datapull.motu.Uri">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.motu.</code><code class="descname">Uri</code><span class="sig-paren">(</span><em>Mopts</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/motu.html#Uri"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.motu.Uri" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <a class="reference internal" href="#geoslurp.datapull.uri.UriBase" title="geoslurp.datapull.uri.UriBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">geoslurp.datapull.uri.UriBase</span></code></a></p>
<dl class="method">
<dt id="geoslurp.datapull.motu.Uri.download">
<code class="descname">download</code><span class="sig-paren">(</span><em>direc</em>, <em>check=False</em>, <em>gzip=False</em>, <em>outfile=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/motu.html#Uri.download"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.motu.Uri.download" title="Permalink to this definition">¶</a></dt>
<dd><p>Download file into directory and possibly check the modification time
:param check : check whether the file needs updating
:param gzip: additionally gzips the file (adds .gz to file name)
:param continueonError (bool): don’t raise an exception when a download error occurrs</p>
</dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.Uri.info">
<code class="descname">info</code><em class="property"> = False</em><a class="headerlink" href="#geoslurp.datapull.motu.Uri.info" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.Uri.kbsize">
<code class="descname">kbsize</code><em class="property"> = 0</em><a class="headerlink" href="#geoslurp.datapull.motu.Uri.kbsize" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.Uri.maxbtdbox">
<code class="descname">maxbtdbox</code><em class="property"> = &lt;geoslurp.tools.Bounds.BtdBox object&gt;</em><a class="headerlink" href="#geoslurp.datapull.motu.Uri.maxbtdbox" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.motu.Uri.maxkbsize">
<code class="descname">maxkbsize</code><em class="property"> = 0</em><a class="headerlink" href="#geoslurp.datapull.motu.Uri.maxkbsize" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.motu.Uri.requestInfo">
<code class="descname">requestInfo</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/motu.html#Uri.requestInfo"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.motu.Uri.requestInfo" title="Permalink to this definition">¶</a></dt>
<dd><p>Request info (modification time, size, datacoverage) on this specific query from the server</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.motu.Uri.updateModTime">
<code class="descname">updateModTime</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/motu.html#Uri.updateModTime"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.motu.Uri.updateModTime" title="Permalink to this definition">¶</a></dt>
<dd><p>Requests data description from the motu service</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.motu.Uri.updateSize">
<code class="descname">updateSize</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/motu.html#Uri.updateSize"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.motu.Uri.updateSize" title="Permalink to this definition">¶</a></dt>
<dd><p>Request information about the size of the query</p>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="module-geoslurp.datapull.rsync">
<span id="geoslurp-datapull-rsync-module"></span><h2>geoslurp.datapull.rsync module<a class="headerlink" href="#module-geoslurp.datapull.rsync" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="geoslurp.datapull.rsync.Crawler">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.rsync.</code><code class="descname">Crawler</code><span class="sig-paren">(</span><em>url</em>, <em>auth</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/rsync.html#Crawler"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.rsync.Crawler" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <a class="reference internal" href="#geoslurp.datapull.crawler.CrawlerBase" title="geoslurp.datapull.crawler.CrawlerBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">geoslurp.datapull.crawler.CrawlerBase</span></code></a></p>
<p>Crawler wrapper around the rsync program calls the linux rsync utility</p>
<dl class="method">
<dt id="geoslurp.datapull.rsync.Crawler.ls">
<code class="descname">ls</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/rsync.html#Crawler.ls"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.rsync.Crawler.ls" title="Permalink to this definition">¶</a></dt>
<dd><p>list remote content (using dry run)</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.rsync.Crawler.parallelDownload">
<code class="descname">parallelDownload</code><span class="sig-paren">(</span><em>outdir</em>, <em>check=False</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/rsync.html#Crawler.parallelDownload"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.rsync.Crawler.parallelDownload" title="Permalink to this definition">¶</a></dt>
<dd><p>Download uris in parallel
:param direc: directory to download to
:param check: Only download when newer or non-existent (default to False)
:param maxconn: amount of parallel downloads to execute
:param continueOnError (bool): keep trying</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.rsync.Crawler.startrsync">
<code class="descname">startrsync</code><span class="sig-paren">(</span><em>cmd</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/rsync.html#Crawler.startrsync"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.rsync.Crawler.startrsync" title="Permalink to this definition">¶</a></dt>
<dd><p>Start rsync and returns the lsit of files as a generator</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.rsync.Crawler.uris">
<code class="descname">uris</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/rsync.html#Crawler.uris"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.rsync.Crawler.uris" title="Permalink to this definition">¶</a></dt>
<dd><p>Generator which returns uri’s to requested datasets</p>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="module-geoslurp.datapull.thredds">
<span id="geoslurp-datapull-thredds-module"></span><h2>geoslurp.datapull.thredds module<a class="headerlink" href="#module-geoslurp.datapull.thredds" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="geoslurp.datapull.thredds.Crawler">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.thredds.</code><code class="descname">Crawler</code><span class="sig-paren">(</span><em>catalogurl</em>, <em>filter=&lt;geoslurp.datapull.thredds.ThreddsFilter object&gt;</em>, <em>followfilter=&lt;geoslurp.datapull.thredds.ThreddsFilter object&gt;</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/thredds.html#Crawler"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.thredds.Crawler" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <a class="reference internal" href="#geoslurp.datapull.crawler.CrawlerBase" title="geoslurp.datapull.crawler.CrawlerBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">geoslurp.datapull.crawler.CrawlerBase</span></code></a></p>
<p>A class to work with an Opendap server</p>
<dl class="staticmethod">
<dt id="geoslurp.datapull.thredds.Crawler.getCatalog">
<em class="property">static </em><code class="descname">getCatalog</code><span class="sig-paren">(</span><em>url</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/thredds.html#Crawler.getCatalog"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.thredds.Crawler.getCatalog" title="Permalink to this definition">¶</a></dt>
<dd><p>Retrieve a catalogue</p>
</dd></dl>

<dl class="staticmethod">
<dt id="geoslurp.datapull.thredds.Crawler.getServices">
<em class="property">static </em><code class="descname">getServices</code><span class="sig-paren">(</span><em>catalog</em>, <em>rooturl</em>, <em>depth=2</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/thredds.html#Crawler.getServices"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.thredds.Crawler.getServices" title="Permalink to this definition">¶</a></dt>
<dd><p>Retrieves the root for serving files over http url from a catalogue</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.thredds.Crawler.setResumePoint">
<code class="descname">setResumePoint</code><span class="sig-paren">(</span><em>filter</em>, <em>followfilt=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/thredds.html#Crawler.setResumePoint"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.thredds.Crawler.setResumePoint" title="Permalink to this definition">¶</a></dt>
<dd><p>Sets the filters after which the normal filters will be applied.</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.thredds.Crawler.unsetResumePoint">
<code class="descname">unsetResumePoint</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/thredds.html#Crawler.unsetResumePoint"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.thredds.Crawler.unsetResumePoint" title="Permalink to this definition">¶</a></dt>
<dd><p>Unset resume point</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.thredds.Crawler.uris">
<code class="descname">uris</code><span class="sig-paren">(</span><em>depth=10</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/thredds.html#Crawler.uris"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.thredds.Crawler.uris" title="Permalink to this definition">¶</a></dt>
<dd><p>Generates a list of threddsURI’s (makes use of xmlitems())</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.thredds.Crawler.xmlitems">
<code class="descname">xmlitems</code><span class="sig-paren">(</span><em>xmlcatalog=None</em>, <em>url=None</em>, <em>depth=10</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/thredds.html#Crawler.xmlitems"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.thredds.Crawler.xmlitems" title="Permalink to this definition">¶</a></dt>
<dd><p>Generator which returns xml nodes which obey a certain filter
Nodes which obey the followFilter will be recursively searched</p>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="geoslurp.datapull.thredds.ThreddsFilter">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.thredds.</code><code class="descname">ThreddsFilter</code><span class="sig-paren">(</span><em>xmltyp='*'</em>, <em>attr=None</em>, <em>regex=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/thredds.html#ThreddsFilter"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.thredds.ThreddsFilter" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p>
<p>Helper class to aid traversing to opendap xml elements</p>
<dl class="method">
<dt id="geoslurp.datapull.thredds.ThreddsFilter.AND">
<code class="descname">AND</code><span class="sig-paren">(</span><em>xmltyp</em>, <em>attr=None</em>, <em>regex=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/thredds.html#ThreddsFilter.AND"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.thredds.ThreddsFilter.AND" title="Permalink to this definition">¶</a></dt>
<dd><p>Provides a method for chaining OR filters</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.thredds.ThreddsFilter.OR">
<code class="descname">OR</code><span class="sig-paren">(</span><em>xmltyp</em>, <em>attr=None</em>, <em>regex=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/thredds.html#ThreddsFilter.OR"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.thredds.ThreddsFilter.OR" title="Permalink to this definition">¶</a></dt>
<dd><p>Provides a method for chaining OR filters</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.thredds.ThreddsFilter.isCatalog">
<code class="descname">isCatalog</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/thredds.html#ThreddsFilter.isCatalog"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.thredds.ThreddsFilter.isCatalog" title="Permalink to this definition">¶</a></dt>
<dd><p>Check if the filter type is a catalogRef</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.thredds.ThreddsFilter.isValid">
<code class="descname">isValid</code><span class="sig-paren">(</span><em>xmlelem</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/thredds.html#ThreddsFilter.isValid"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.thredds.ThreddsFilter.isValid" title="Permalink to this definition">¶</a></dt>
<dd><p>Filter xmlelem on  attributes</p>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="geoslurp.datapull.thredds.Uri">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.thredds.</code><code class="descname">Uri</code><span class="sig-paren">(</span><em>dataxml</em>, <em>services</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/thredds.html#Uri"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.thredds.Uri" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <a class="reference internal" href="#geoslurp.datapull.uri.UriBase" title="geoslurp.datapull.uri.UriBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">geoslurp.datapull.uri.UriBase</span></code></a></p>
<p>Thredds URI class</p>
<dl class="attribute">
<dt id="geoslurp.datapull.thredds.Uri.opendap">
<code class="descname">opendap</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.thredds.Uri.opendap" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.thredds.Uri.suburl">
<code class="descname">suburl</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.thredds.Uri.suburl" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

</dd></dl>

<dl class="function">
<dt id="geoslurp.datapull.thredds.getAttrib">
<code class="descclassname">geoslurp.datapull.thredds.</code><code class="descname">getAttrib</code><span class="sig-paren">(</span><em>xml</em>, <em>regex</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/thredds.html#getAttrib"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.thredds.getAttrib" title="Permalink to this definition">¶</a></dt>
<dd><p>Search in xml attributes based on a regex</p>
</dd></dl>

<dl class="function">
<dt id="geoslurp.datapull.thredds.getDate">
<code class="descclassname">geoslurp.datapull.thredds.</code><code class="descname">getDate</code><span class="sig-paren">(</span><em>xml</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/thredds.html#getDate"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.thredds.getDate" title="Permalink to this definition">¶</a></dt>
<dd><p>extracts the date from a dataset element</p>
</dd></dl>

<dl class="function">
<dt id="geoslurp.datapull.thredds.getTagEnding">
<code class="descclassname">geoslurp.datapull.thredds.</code><code class="descname">getTagEnding</code><span class="sig-paren">(</span><em>xml</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/thredds.html#getTagEnding"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.thredds.getTagEnding" title="Permalink to this definition">¶</a></dt>
<dd><p>Strip the leading junk ({…}) from a tag</p>
</dd></dl>

<dl class="function">
<dt id="geoslurp.datapull.thredds.gethref">
<code class="descclassname">geoslurp.datapull.thredds.</code><code class="descname">gethref</code><span class="sig-paren">(</span><em>input</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/thredds.html#gethref"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.thredds.gethref" title="Permalink to this definition">¶</a></dt>
<dd><p>small function to extract a href link from a dictionary</p>
</dd></dl>

</div>
<div class="section" id="module-geoslurp.datapull.uri">
<span id="geoslurp-datapull-uri-module"></span><h2>geoslurp.datapull.uri module<a class="headerlink" href="#module-geoslurp.datapull.uri" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="geoslurp.datapull.uri.UriBase">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.uri.</code><code class="descname">UriBase</code><span class="sig-paren">(</span><em>url</em>, <em>lastmod=None</em>, <em>auth=None</em>, <em>subdirs=''</em>, <em>basedir=''</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/uri.html#UriBase"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.uri.UriBase" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></p>
<p>Base class to store uri resource</p>
<dl class="attribute">
<dt id="geoslurp.datapull.uri.UriBase.auth">
<code class="descname">auth</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.uri.UriBase.auth" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.uri.UriBase.basedir">
<code class="descname">basedir</code><em class="property"> = ''</em><a class="headerlink" href="#geoslurp.datapull.uri.UriBase.basedir" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.uri.UriBase.buffer">
<code class="descname">buffer</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/uri.html#UriBase.buffer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.uri.UriBase.buffer" title="Permalink to this definition">¶</a></dt>
<dd><p>Download file into a buffer (default uses curl)</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.uri.UriBase.download">
<code class="descname">download</code><span class="sig-paren">(</span><em>direc</em>, <em>check=False</em>, <em>gzip=False</em>, <em>gunzip=False</em>, <em>outfile=None</em>, <em>continueonError=False</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/uri.html#UriBase.download"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.uri.UriBase.download" title="Permalink to this definition">¶</a></dt>
<dd><p>Download file into directory and possibly check the modification time
:param check : check whether the file needs updating
:param gzip: additionally gzips the file (adds .gz to file name)
:param continueonError (bool): don’t raise an exception when a download error occurrs</p>
</dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.uri.UriBase.lastmod">
<code class="descname">lastmod</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.uri.UriBase.lastmod" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.uri.UriBase.subdirs">
<code class="descname">subdirs</code><em class="property"> = ''</em><a class="headerlink" href="#geoslurp.datapull.uri.UriBase.subdirs" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.uri.UriBase.updateModTime">
<code class="descname">updateModTime</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/uri.html#UriBase.updateModTime"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.uri.UriBase.updateModTime" title="Permalink to this definition">¶</a></dt>
<dd><p>Tries to retrieve the last modification time of a file
Note: his is often not supported by the server</p>
</dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.uri.UriBase.url">
<code class="descname">url</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.uri.UriBase.url" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

</dd></dl>

<dl class="class">
<dt id="geoslurp.datapull.uri.UriFile">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.uri.</code><code class="descname">UriFile</code><span class="sig-paren">(</span><em>url</em>, <em>lastmod=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/uri.html#UriFile"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.uri.UriFile" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <a class="reference internal" href="#geoslurp.datapull.uri.UriBase" title="geoslurp.datapull.uri.UriBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">geoslurp.datapull.uri.UriBase</span></code></a></p>
<dl class="method">
<dt id="geoslurp.datapull.uri.UriFile.buffer">
<code class="descname">buffer</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/uri.html#UriFile.buffer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.uri.UriFile.buffer" title="Permalink to this definition">¶</a></dt>
<dd><p>Download file into a buffer (default uses curl)</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.uri.UriFile.updateModTime">
<code class="descname">updateModTime</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/uri.html#UriFile.updateModTime"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.uri.UriFile.updateModTime" title="Permalink to this definition">¶</a></dt>
<dd><p>Tries to retrieve the last modification time of a file
Note: his is often not supported by the server</p>
</dd></dl>

</dd></dl>

<dl class="function">
<dt id="geoslurp.datapull.uri.curlDownload">
<code class="descclassname">geoslurp.datapull.uri.</code><code class="descname">curlDownload</code><span class="sig-paren">(</span><em>url</em>, <em>fileorfid</em>, <em>mtime=None</em>, <em>gzip=False</em>, <em>gunzip=False</em>, <em>auth=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/uri.html#curlDownload"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.uri.curlDownload" title="Permalink to this definition">¶</a></dt>
<dd><p>Download  the content of an url to an open file or buffer using pycurl
:param url: url to download from
:param fileorfid: filename or open file or buffer
:param mtimee: explicitly set the modification time to this (usefull when modification times are not supported
b the server)
:param gzip: additionally gzip the file on disk (note this routine does not append *.gz to the file name)
:param gunzip: automatically gunzip the downloaded file
:param auth: supply authentification data (user and passw)
:return: modification time of remote file</p>
</dd></dl>

<dl class="function">
<dt id="geoslurp.datapull.uri.findFiles">
<code class="descclassname">geoslurp.datapull.uri.</code><code class="descname">findFiles</code><span class="sig-paren">(</span><em>dir</em>, <em>pattern</em>, <em>since=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/uri.html#findFiles"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.uri.findFiles" title="Permalink to this definition">¶</a></dt>
<dd><p>Generator to recursively search adirecctor (returns a generator)</p>
</dd></dl>

<dl class="function">
<dt id="geoslurp.datapull.uri.setFtime">
<code class="descclassname">geoslurp.datapull.uri.</code><code class="descname">setFtime</code><span class="sig-paren">(</span><em>file</em>, <em>modTime=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/uri.html#setFtime"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.uri.setFtime" title="Permalink to this definition">¶</a></dt>
<dd><p>change modification and access time of a file</p>
</dd></dl>

<dl class="function">
<dt id="geoslurp.datapull.uri.timeFromStamp">
<code class="descclassname">geoslurp.datapull.uri.</code><code class="descname">timeFromStamp</code><span class="sig-paren">(</span><em>stamp</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/uri.html#timeFromStamp"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.uri.timeFromStamp" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

</div>
<div class="section" id="module-geoslurp.datapull.webdav">
<span id="geoslurp-datapull-webdav-module"></span><h2>geoslurp.datapull.webdav module<a class="headerlink" href="#module-geoslurp.datapull.webdav" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="geoslurp.datapull.webdav.Crawler">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.webdav.</code><code class="descname">Crawler</code><span class="sig-paren">(</span><em>rooturl</em>, <em>pattern</em>, <em>auth</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/webdav.html#Crawler"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.webdav.Crawler" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <a class="reference internal" href="#geoslurp.datapull.crawler.CrawlerBase" title="geoslurp.datapull.crawler.CrawlerBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">geoslurp.datapull.crawler.CrawlerBase</span></code></a></p>
<p>Webdav Crawler</p>
<dl class="attribute">
<dt id="geoslurp.datapull.webdav.Crawler.pattern">
<code class="descname">pattern</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.webdav.Crawler.pattern" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.webdav.Crawler.uris">
<code class="descname">uris</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/webdav.html#Crawler.uris"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.webdav.Crawler.uris" title="Permalink to this definition">¶</a></dt>
<dd><p>Generator which returns uri’s to requested datasets</p>
</dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.webdav.Crawler.webdavroot">
<code class="descname">webdavroot</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.webdav.Crawler.webdavroot" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

</dd></dl>

<dl class="class">
<dt id="geoslurp.datapull.webdav.Uri">
<em class="property">class </em><code class="descclassname">geoslurp.datapull.webdav.</code><code class="descname">Uri</code><span class="sig-paren">(</span><em>rooturl</em>, <em>lastmod=None</em>, <em>auth=None</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/webdav.html#Uri"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.webdav.Uri" title="Permalink to this definition">¶</a></dt>
<dd><p>Bases: <a class="reference internal" href="#geoslurp.datapull.uri.UriBase" title="geoslurp.datapull.uri.UriBase"><code class="xref py py-class docutils literal notranslate"><span class="pre">geoslurp.datapull.uri.UriBase</span></code></a></p>
<p>“Webdav URI</p>
<dl class="method">
<dt id="geoslurp.datapull.webdav.Uri.connect">
<code class="descname">connect</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/webdav.html#Uri.connect"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.webdav.Uri.connect" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.webdav.Uri.download">
<code class="descname">download</code><span class="sig-paren">(</span><em>direc</em>, <em>check=False</em>, <em>gzip=False</em>, <em>outfile=None</em>, <em>continueonError=False</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/webdav.html#Uri.download"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.webdav.Uri.download" title="Permalink to this definition">¶</a></dt>
<dd><p>Download file into directory and possibly check the modification time
:param check : check whether the file needs updating
:param gzip: additionally gzips the file (adds .gz to file name)
:param continueonError (bool): don’t raise an exception when a download error occurrs</p>
</dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.webdav.Uri.ls">
<code class="descname">ls</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/webdav.html#Uri.ls"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.webdav.Uri.ls" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="method">
<dt id="geoslurp.datapull.webdav.Uri.subUri">
<code class="descname">subUri</code><span class="sig-paren">(</span><em>remf</em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/geoslurp/datapull/webdav.html#Uri.subUri"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#geoslurp.datapull.webdav.Uri.subUri" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns a webdav URI derived from this one</p>
</dd></dl>

<dl class="attribute">
<dt id="geoslurp.datapull.webdav.Uri.webdav">
<code class="descname">webdav</code><em class="property"> = None</em><a class="headerlink" href="#geoslurp.datapull.webdav.Uri.webdav" title="Permalink to this definition">¶</a></dt>
<dd></dd></dl>

</dd></dl>

</div>
</div>


           </div>
           <div class="articleComments">
            
           </div>
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="geoslurp.dataset.html" class="btn btn-neutral float-right" title="geoslurp.dataset package" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="geoslurp.config.html" class="btn btn-neutral" title="geoslurp.config package" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        &copy; Copyright 2020, Roelof Rietbroek.

    </p>
  </div>
  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  


  

    <script type="text/javascript">
        var DOCUMENTATION_OPTIONS = {
            URL_ROOT:'../',
            VERSION:'v1.1.0',
            COLLAPSE_INDEX:false,
            FILE_SUFFIX:'.html',
            HAS_SOURCE:  true,
            SOURCELINK_SUFFIX: '.txt'
        };
    </script>
      <script type="text/javascript" src="../_static/jquery.js"></script>
      <script type="text/javascript" src="../_static/underscore.js"></script>
      <script type="text/javascript" src="../_static/doctools.js"></script>

  

  
  
    <script type="text/javascript" src="../_static/js/theme.js"></script>
  

  
  
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.StickyNav.enable();
      });
  </script>
   

</body>
</html>